import requests,time
from datetime import datetime
from lxml import etree
from tinydb import where
from common import UID,HandleTmpList,parseContentToName,ProductToGroup,sess
SPIDERNAME='工信部'
def parseGXBArtPubTime(time_string):
    #1609830015678
    timeStamp = float(time_string)/1000
    timeArray = time.localtime(timeStamp)
    otherStyleTime = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
    return otherStyleTime   # 2013-10-10 23:40:00

def parseGXBContent(url):
    r=sess.get(url)
    selector=etree.HTML(r.text)
    ele=selector.cssselect('div#con_con')[0]
    content=ele.xpath('string(.)')
    return content.strip()

def getGXBArticleList(articleCol,BeCrawledUrlList):
    #政策文件
    url='https://www.miit.gov.cn/search-front-server/api/search/info'
    param={
        'websiteid': 110000000000000,
        'scope': 'basic',
        'q':'',   #搜索关键字参数
        'pg': 10,
        'cateid': 57,
        'pos': 'title_text,infocontent,titlepy',
        '_cus_eq_typename': '',
        '_cus_eq_publishgroupname': '',
        '_cus_eq_themename': '',
        'begin': '',
        'end': '',
        'dateField': 'deploytime',
        'selectFields': 'title,content,deploytime,_index,url,cdate,infoextends,infocontentattribute,columnname,filenumbername,publishgroupname,publishtime,metaid,bexxgk,columnid,xxgkextend1,xxgkextend2,themename,typename,indexcode,createdate',
        'group': 'distinct',
        'highlightConfigs': '[{"field":"infocontent","numberOfFragments":2,"fragmentOffset":0,"fragmentSize":30,"noMatchSize":145}]',
        'highlightFields': 'title_text,infocontent,webid',
        'level': 6,
        'sortFields': '[{"name":"deploytime","type":"desc"}]',
        'p': 1,
    }
    r=sess.get(url,params=param)
    temp_article_ls=[]
    for item in r.json()['data']['searchResult']['dataResults']:
        # print(item['groupData'][0]['data']['title'])
        articleUrl='https://www.miit.gov.cn'+item['groupData'][0]['data']['url']
        if len(articleCol.search(where('url')==articleUrl)):break
        title=item['groupData'][0]['data']['title']
        publicTime=parseGXBArtPubTime(item['groupData'][0]['data']['publishtime'])
        temp_dict={'tags':SPIDERNAME,'score':0,'uid':UID()}
        temp_dict['title']=title.strip()
        temp_dict['articleFrom']=SPIDERNAME
        temp_dict['url']=articleUrl.strip()
        temp_dict['publicTime']=publicTime.strip()
        #文章内容
        content=parseGXBContent(articleUrl)
        temp_dict['content']=content
        #定文章所属期货品种,板块
        n=parseContentToName(title+content)
        if n:
            print(SPIDERNAME,'   ',title,'     ',n)
            temp_dict['product_name']=n
            temp_dict['group']=ProductToGroup[n]
        else:
            print("………………………………未找到品种名称，可能异常")
            temp_dict['product_name']=''
            temp_dict['group']=''

        temp_article_ls.append(temp_dict)

    #注意缩进不要错
    HandleTmpList(temp_article_ls,articleCol,SPIDERNAME)


    #政策解读